#!/bin/bash
#TODO: Warn about file (not directory) symlinks so they don't get left behind in the process
#TODO: Option Give special treatment to video/audio files? mm directory? archive them? what about bought ones?
#TODO: Option Give special treatment to files whose mtime is before or equal the ctime, which means most likely they are reference only files
#TODO: If .doc-now, etc exist, but are empty, what to do?

#Exit if  any command returns non-true, and don't allow unset variables
set -eu
#Fail as early as possible in the pipe
set -o pipefail


ROOT=$HOME/tmp/test


#Size limit is the combination of NOW and NICENOW
SIZELIMIT=10G

##########################
#Criteria
##########################
#Files will be moved from NOW_DIR to NICENOW_DIR if they are older than this many days
NICENOW_DAYS=100  

#Files that are larger than NICENOW_SIZE (even if they are newer than NICENOW_DAYS)
#will also be moved from NOW_DIR to NICENOW_DIR
#Use k for kilobytes, M for Megabytes and G for Gigabytes: e.g. 100M, 10G or 10k
NICENOW_ABOVESIZE=200M


##########################
#Other Options
##########################
PREPAREONLY="no"
QUIET="no"	#If yes, do not notify the user, or print summaries, but still log INFO, ERROR and WARNINGS, show WARNINGS and ERRORS on stderr
IGNORE=".*" 	#Pattern of files to ignore
FAKE="no" 	#Don't execute commands that change the filesystem, but simply show them on stdout
# Values for VERBOSE: 
#	1 - Show total of files moved, and summaries;
#	2 - echo commands that will modify the filesystem (use FAKE to prevent their execution)
VERBOSE=1
SUMMARY="yes"	#Do a summary report instead of a detailed one
UNDECLUTTER="no"	# Return directory to its original condition before decluttering

#############################################################################################################
#############################################################################################################
#Internal variables
NOTIFY="`which notify-send` -i /usr/share/notify-osd/icons/gnome/scalable/status/notification-device-usb.svg declutter" || `which aosd_cat` -n 20
MSG=""
bold="`tput bold`" 
nobold="`tput sgr0`"
red="`tput setaf 1`"
nored="`tput setaf 9`"
green="`tput setaf 2`"
nogreen="`tput setaf 9`"
ul="`tput smul`"
noul="`tput rmul`"

#Utility functions
log() 		{ logger -i -t declutter "$@"; }
logInfo() 	{ log "INFO: $@"; echostderr "${bold}INFO:${nobold} $@"; }
logError()	{ log "ERROR: $@"; echostderr "${red}ERROR:${nored} $@"; }
logWarning() 	{ log "WARNING: $@";  echostderr "${green}WARNING:${nogreen} $@"; }
echostderr()	{ echo  -e "$@" 1>&2; } 
#Notify using OSD or notify daemons
notify()	{ [ "$QUIET" = "yes" ] || $NOTIFY "$@"; logInfo "$@"; }
#Convert human sizes (10G, 5M or 10K) to bytes
kmg2b()		{ echo "$1" | sed -e 's/\([0-9.]\+\)[Gg]/\1*1073741824/
			              s/\([0-9.]\+\)[Mm]/\1*1048576/
			              s/\([0-9.]\+\)[Kk]/\1*1024/
			              s/\([0-9.]\+\) *\(bytes\|[bB]\)/\1/' | bc -l | xargs printf "%1.0f"; }
#Convert bytes to human sizes (e.g. 10G, 8M, 10K, 5 bytes)
b2kmg()		{ if [ $1 -lt  1024 ]; then echo "$1 bytes";
		  elif [ $1 -lt  1048576 ]; then echo "$1/1024" | bc -l | xargs printf "%1.1fK"; 
	       elif [ $1 -lt 1073741824 ]; then  echo "$1/1048576" | bc -l | xargs printf "%1.1fM";
		  else echo "$1/1073741824" | bc -l | xargs printf "%1.1fG"; fi; }
#Size of directory in bytes
dirsize()	{ du -sk "$ROOT/$1" | cut -f 1 | xargs expr 1024 \* ; }

#Commands that change the filesystem should be exexuted by saying dothis command (e.g. "mv foo bar" should be "dothis mv foo bar")
#Prints command on stderr before execution if VERBOSE>1 and QUIET is not set.
#If FAKE is yes, then command is not executed. 
dothis()	{ [ "$QUIET" = "yes" -o "$VERBOSE" -lt 2 ] || echostderr "$@"; [ "$FAKE" = "yes" ] || "$@"; }

#Move a file making sure that the file does not exist beforhand, and creating the parent directories if neccessary.
move()		{ [ -e "$2" ] && { logError "Moving '$1' to '$2'. But '$2' already exists, aborting."; exit 8; }
		  dothis mkdir -p "`dirname "$2"`";
		  dothis mv -n "$1" "$2"; }

version()	{ echo "declutter - Clean up directories based on age and size of files - v0.1"; }

mount_unionfs()
{
   #Mount unionfs to have one 'view' of NOW, NICENOW, ARK and ATTIC
   unionfs_fuse=`which unionfs-fuse`
   if [ ! -x $unionfs_fuse ]; then 
      # Another option is to simply make a symbolic link to nicenow, give less advantages.
      logError "unionfs-fuse is required."
      exit 6;
   else
      #return if already mounted, otherwise mount it.
      mount | grep "unionfs-fuse.*$ROOT/$DIR" > /dev/null && return 0
      [ ! -e "$DIR" -a -e "$NOW_DIR" -a -e "$NICENOW_DIR" -a -e "$ARK_DIR" ] && mkdir "$DIR"
      if [ -z "`ls -A "$DIR"`" ]; then
	 $unionfs_fuse -o cow,use_ino "$NOW_DIR=RW:$NICENOW_DIR=RO:$ARK_DIR=RO" "$DIR" && logInfo "Unionfs mounted on '$DIR'.";
      else
	 logError "'$DIR' is not empty, please empty it before trying to mount it."
	 exit 18;
      fi
   fi
}

#Make sure directories exist, set DIR,NICE_DIR,NOW_DIR variables, mount unionfs, etc
prepare()
{
   DIR="$1"
   NOW_DIR=".$1-now"
   NICENOW_DIR=".$1-nicenow"
   ARK_DIR=".$1-ark"
   #FIXME ATTIC_DIR=".$1-attic"

   [ "$QUIET" = "yes" ] || version
   [ "$QUIET" = "yes" ] || logInfo "Decluttering '$DIR'."
   cd "$ROOT" || { logError "Unable to cd to '$ROOT'. Good bye."; exit 3; }
   
   if [ ! -e "$DIR" ]; then
      if [ ! -e "$NOW_DIR" ]; then
         logError "'$ROOT/$DIR' does not exist. Good bye.";
         exit 4;
      else
	 logInfo "Seems there is an orphaned '$NOW_DIR' directory. Taking corrective action.";
      fi
   fi

   #Set the variable 'moved' to would have been moved if FAKE is yes.
   MOVED="moved"
   [ "$FAKE" = "yes" ] && MOVED="\"moved\""

   #Move $DIR to $NOW_DIR if it is a directory with data, and not a unionfs mount or a symbolic link
   if [ "$UNDECLUTTER" != "yes" ] && ! mount | grep "unionfs-fuse.*$ROOT/$DIR" > /dev/null && [ -d "$DIR" -a ! -L "$DIR" ] && [ "n" != "n`ls -A $DIR`" ]; then 
   { 
      [ "$FAKE" = "yes" ] && { logError "'Fake' option is set, so I can't move '$DIR' or create necessary directories.\nUse 'Prepare only' option if you don't want to move files."; exit 10; }
      read -t 50 -p "'$DIR' already exists. declutter will move it to '$NOW_DIR' and create the other necessary directories, do you want do continue (y/n)?"
      if [ $REPLY = "y" ]; then
	 [ -e "$NOW_DIR" ] && { logError "'$NOW_DIR' already exists, please delete it or move it somewhere else."; exit 11; }
	 [ ! -e "$NOW_DIR" ] && mv -n "$DIR" "$NOW_DIR"
      else
         echo "Good bye!"
	 logInfo "'$DIR' already exists, left it alone because the user said so. Good bye."
	 exit 2;
      fi
   }
   fi

   if [ "$UNDECLUTTER" != "yes" ]; then
   #Create all needed directories
      if [ ! -d $NOW_DIR ]; then  logInfo "$NOW_DIR is not a directory or does not exist, creating it."; mkdir "$NOW_DIR";  fi
      if [ ! -d $NICENOW_DIR ]; then  logInfo "$NICENOW_DIR is not a directory or does not exist, creating it."; mkdir "$NICENOW_DIR"; fi
      if [ ! -d $ARK_DIR ]; then  logInfo "$ARK_DIR is not a directory or does not exist, creating it.";  mkdir "$ARK_DIR"; fi;
   fi
   
   #Mount unionfs filesystem, even if we are undecluttering (in case there are orphaned NOW_DIR, NICENOW_DIR and ARK_DIR
   mount_unionfs

   #Exit if prepare only flag is set
   if [ "$PREPAREONLY" = "yes" ]; then
      exit 0;
   fi
}

# move_from_to srcdir1 srcdir2... targetdir
#Move list of files (read from stdin) from the source directories to the target directory (last positional parameter)
#Outputs a message to stdout stating the number of files moved, their size and the target directory.
move_from_to()
{
   #Escape sed regular expression characters present in directory name
   #to prevent nasty errors.
   escape_characters()
   {
      local r=""
      for ((p=1;p<$#;p++)); do
	 r+="`echo "${!p}" | sed -e 's#[$^*.[]#\\\&#g'`\|";
      done
      r=${r:0:${#r}-2} #Delete the last trainling \|
      echo "$r"
   }

   local count=0
   local totalsize=0
   local from="`escape_characters "$@"`" #source directories separated by \| to use in regex within loop
   local to="${!#}" #take the last positional argument (the target dir)
   
   #Each line of stdin contains the size and the name of the file to move separated by \t
   #This loop moves each file to the target directory (the last positional argument)
   #and reports the total number of files moved and their size.
   while read size src; do
      local dest="`echo "$src" | sed -e "s#\($from\)/\(.*\)#$to/\2#"`"
      move "$src" "$dest" && { [ "$VERBOSE" -gt 1 ] && logInfo "moved '$src' ==>> '$dest'"; }
      let ++count
      let totalsize+=$size;
   done
   [ "$VERBOSE" -gt 0 ] && echo -e "$count:`b2kmg $totalsize`:'${@:1:$(($#-1))}':'$to'"
}

# clean_biggest_in_NICENOW_and_NOW sizeToMove
#Takes files from NICENOW_DIR (and NOW_DIR if needed) and moves them to ARK_DIR
#until the size is reduced by at least "sizeToMove" bytes.
clean_biggest_in_NICENOW_and_NOW()
{
   let totalsizefound=0
   
   #From the list in stdin, keep only enough files to reach a size which is at least "sizeToMove" bytes.
   pick_needed()
   {
      while read size file && [ $totalsizefound -lt $size2move ]; do
	 let totalsizefound+=size
	 echo -e "$size\t$file"
      done
      [ "$VERBOSE" -gt 0 ] && logInfo "Found `b2kmg $totalsizefound` to move."      
   }

   [ -d $NOW_DIR ] || { logError "$NOW_DIR is not a directory or does not exist. Aborting."; exit 1; }
   [ -d $NICENOW_DIR ] || { logError "$NICENOW_DIR is not a directory or does not exist. Aborting."; exit 1; }
   [ -d $ARK_DIR ] || { logError "$ARK_DIR is not a directory or does not exist. Aborting."; exit 1; }
   
   local size2move=$1
   [ "$VERBOSE" -gt 0 ] && logInfo "Seeking to move `b2kmg $size2move` from '$NICENOW_DIR'."
   cd "$ROOT"
   # Get files in $NOW_DIR and $NICENOW_DIR and sort them by size (descending) to pick the ones we need to bring 
   # the total size below $SIZELIMIT - read output message from stdout of move_from_to
   find  $NICENOW_DIR \( -type f \) ! -name "$IGNORE" -printf "%s\t%p\n" | sort -gr | pick_needed | move_from_to "$NICENOW_DIR" "$ARK_DIR"
   
   #If there were not enough files in NICENOW_DIR use NOWDIR
   if [  $size2move -gt $totalsizefound  ]; then
      [ "$VERBOSE" -gt 0 ] && logInfo "'$NICENOW_DIR' did not have enough files reduce the size by `b2kmg $size2move`. Trying also from '$NOW_DIR'."
      let size2move-=$totalsizefound
      [ "$VERBOSE" -gt 0 ] && logInfo "We $MOVED `b2kmg $totalsizefound` from '$NICENOW_DIR'. Seeking `b2kmg $size2move` from '$NOW_DIR'."
      find  $NOW_DIR \( -type f \) ! -name "$IGNORE" -printf "%s\t%p\n" | sort -gr | pick_needed | move_from_to "$NOW_DIR" "$ARK_DIR"
   fi
}

#clean_NOW
#Looks for old or big files in NOW_DIR and moves them to NICENOW_DIR
#A file is old if it is older than NICENOW_DAYS,
#and big if it is bigger than NICENOW_ABOVESIZE
#files with a name that matches the glob pattern listed in IGNORE are ignored.
clean_NOW()
{
   [ -d $NOW_DIR ] || { logError "$NOW_DIR is not a directory or does not exist. Aborting."; exit 1; } 
   [ -d $NICENOW_DIR ] || { logError "$NICENOW_DIR is not a directory or does not exist. Aborting."; exit 1; }

   CRITERIA="-mtime +$NICENOW_DAYS -o -size +$NICENOW_ABOVESIZE"
   cd "$ROOT"
   #Find the files, sort them by size, and move them to NICENOW_DIR
   find $NOW_DIR \( -type f \) ! -name "$IGNORE" \( $CRITERIA \) -printf "%s\t%p\n" | sort -gr | move_from_to "$NOW_DIR" "$NICENOW_DIR"   
}

#clean_NICENOW
#If the size of NICENOW_DIR and NOW_DIR together is greater than SIZELIMIT then
#move files from NICENOW to ARK_DIR until the total size is below SIZELIMIT
clean_NICENOW()
{
   # 1. If NOW is above SIZELIMIT, move biggest files to NICENOW
   local size=`dirsize "$NOW_DIR"`
   let size+=`dirsize "$NICENOW_DIR"`
   local size2move=$(($size - `kmg2b $SIZELIMIT`))
   if [ $size -gt `kmg2b $SIZELIMIT` ]; then
      [ "$VERBOSE" -gt 0 ] && logInfo "'$NOW_DIR' and '$NICENOW_DIR' are too big: `b2kmg $size`. Moving files to '$ARK_DIR' to reach $SIZELIMIT.";
      clean_biggest_in_NICENOW_and_NOW "$size2move"
   fi
}
report()
{
   local tcount=0; local tsize=0; local count=0; local size=0; local from=""; local to=""
   OLDIFS="$IFS"
   IFS=":"
   while read count size from to && [ -n "$count" ]; do
      if [ "$SUMMARY" = "yes" ]; then
	 let tcount+=$count
	 let tsize+=`kmg2b $size`;
      else
	 echo -e "$count files ($size) $MOVED:\n  (  $from "$'\U21D2'" $to)";
      fi;
   done
   [ "$SUMMARY" = "yes" ] && echo "$tcount (`b2kmg $tsize`) files moved."
   IFS="$OLDIFS"
}

declutter()
{
   local oldsize=`dirsize "$NOW_DIR"`
   MSG+="`clean_NOW`\n"
   MSG+="`clean_NICENOW`\n"
   local newsize=`dirsize "$NOW_DIR"`
   local freedsize=$(($oldsize - $newsize))
      
   MSG="`echo -e "$MSG" | report`"
   notify "${MSG}\n'$NOW_DIR' is `b2kmg $newsize` (freed `b2kmg $freedsize`)."
}

move_empty_dirs_to()
{

   local count=0
   local to="$1"
   
   [ -n "`ls -A "$1"`" ] && { logError "'$1' is not empty, will not move it to '$2'"; exit 17; }
   
   #Each line of stdin contains the size and the name of the directory to move separated by \t
   #This loop moves each directory to the target directory (they come sorted by ascending depth in the tree)
   #and reports the total number of directories moved.
   while read depth size src; do
      local dest="`echo "$src" | sed -e "s#\([^/]\+\)/\(.*/\)[^/]\+#$to/\2#"`"
      [ ! -e "$dest" ] && dothis mkdir -p "$dest"
      dothis mv "$src" "$dest" && { [ "$VERBOSE" -gt 1 ] && logInfo "moved '$src' ==>> '$dest'"; }
      let ++count
   done
   [ "$VERBOSE" -gt 0 ] && echo -e "Moved $count empty directories."
}

undeclutter()
{
   #This function:
   #1. Unmount the filesystem on $DIR
   #2. Copy/move all files from now, nicenow, ark to the original directory ($DIR)
   #3. delete now, nicenow, ark and attic directories.
   
   cd "$ROOT"
   local oldls="`ls -Ras "$DIR"`"
   local size="`dirsize "$DIR"`"
   local freesize="`df  -B 1024 "$DIR" | awk '{print $4}' | tail -1`"
   let freesize=$freesize*1024  	#Convert number into bytes
   let neededsize=$size*2+$freesize*10/100
   
   #Unmount direcoty if it is mounted
   if mount | grep "unionfs-fuse.*$ROOT/$DIR" > /dev/null; then
      umount="`which fusermount`" || { logError "fusermount not found. Aborting."; exit 13; }
      $umount -u "$DIR";
   fi
   
   #Make sure $DIR is empty
   if [ "n`ls -A "$DIR"`" != "n" ]; then
      logError "'$NOW_DIR' is not empty, can not un-declutter. Aborting."
      exit 14;
   fi
      
   #Move files from NOW_DIR, NICENOW_DIR and ARK_DIR back to $DIR
   #Move empty directories first, then files.
   logInfo "Moving all files in '$NOW_DIR','$NICENOW_DIR', and '$ARK_DIR' back to '$DIR'."
   MSG="`find "$NOW_DIR" "$NICENOW_DIR" "$ARK_DIR" -mindepth 1 -type d -empty  -printf "%d\t%s\t%p\n" | sort -g | move_empty_dirs_to "$DIR"`\n"
   MSG+="`find "$NOW_DIR" "$NICENOW_DIR" "$ARK_DIR" -type f -printf "%s\t%p\n" | move_from_to "$NOW_DIR" "$NICENOW_DIR" "$ARK_DIR" "$DIR" | report`\n"

      
   #Compare the new list of files with the old one to make sure it is consitent.
   local newls="`ls -Ras "$DIR"`"
   mtxt="mismatch.txt"
   if [ "$oldls" != "$newls" ] && [ "$FAKE" != "yes" ]; then
      logError "There was a problem un-decluttering '$DIR'. File listings don't match after move. Difference stored in mismatch.txt.\nYou will have to do it manually. No directories deleted."
      echo "================= Before move =================" > "$mtxt"
      echo "$oldls" | tee 1.err.declutter >> "$mtxt"
      echo "================= After move =================" >> "$mtxt"
      echo "$newls" | tee 2.err.declutter >> "$mtxt"
      echo "================= Difference =================" >> "$mtxt"
      diff -u 1.err.declutter 2.err.declutter >> "$mtxt"
      [ -x `which kompare` ] && kompare 1.err.declutter 2.err.declutter&
      exit 14;
   else
      dothis rm -rf "$NOW_DIR" "$NICENOW_DIR" "$ARK_DIR"
      notify "$MSG\n'$NOW_DIR','$NICENOW_DIR', and '$ARK_DIR''$DIR' have been deleted. '$DIR' restored.";
   fi
   
}

usage()
{
   version
   echo -e "Usage: declutter [-pfqru] [-l ${ul}sizelimit${noul}] [-d ${ul}days${noul}] [-s ${ul}filesize${noul}] [-i ${ul}pattern${noul}] [-v ${ul}level${noul}] ${ul}directory${noul}";
   echo -e "   -p           - Only prepare directories, and mount unionfs filesystem,"
   echo -e "                  but don't do anything else."
   echo -e "   -f           - Fake. do not move any files, use with -v 2 to show the"
   echo -e "                  commands that would have been executed."
   echo -e "                  Because no files are moved, the numbers and "
   echo -e "                  the actual execution might be a little different."
   echo -e "   -q           - Quiet."
   echo -e "   -r           - Do a more detailed report instead of a summary report."
   echo -e "   -u           - Undeclutter. Leave directory as it was before declutter."
   echo -e "   -l ${ul}sizelimit${noul} - move fiels if 'now' and 'nice to have now' are beyond this size limit."
   echo -e "                  Files moved to 'archive'. Default is 10G (G,K, or M can be used as units)."
   echo -e "   -d ${ul}days${noul}      - move files in 'now' that are older than this number of days."
   echo -e "                  Default is 100."
   echo -e "   -s ${ul}size${noul}      - move files in 'now' that are bigger than this size.( e.g. 10G,20M, 10K)"
   echo -e "   -i ${ul}pattern${noul}   - Ignore files whose name maches pattern."
   echo -e "   -v ${ul}level${noul}     - Verbosity level, 2 is highest."
   echo -e "   ${ul}directoty${noul}    - This is the directory to declutter."
}

while getopts ":pfqruml:d:s:i:v:" opt; do
case $opt in
   p)
      PREPAREONLY="yes"
      ;;
   f)
      FAKE="yes"
      ;;
   q)
      QUIET="yes"
      ;;
   r)
      SUMMARY="no"
      ;;
   u)
      UNDECLUTTER="yes"
      ;;
   m)
      #TODO mount unionfs
      exit
      ;;
   l)
      SIZELIMIT="$OPTARG"
      ;;
   d)
      NICENOW_DAYS="$OPTARG"
      ;;
   s)
      NICENOW_ABOVESIZE="$OPTARG"
      ;;
   i)
      IGNORE="$OPTARG"
      ;;
   v)
      VERBOSE="$OPTARG"
      ;;
   \?)
      logError "Invalid option: -$OPTARG."
      usage
      exit 12
      ;;
   :)
      logError "Missing argument for option -$OPTARG."
      usage
      exit 12
      ;;
esac
done
shift $(($OPTIND-1))


[  $# -eq 0 ] &&  { logError "Please specify directory"; usage; }

#Declutter each directory in the command line
for d in "$@"; do
   prepare $d
   if [ "$UNDECLUTTER" != "yes" ]; then
      [ "$PREPAREONLY" = "yes" ] || declutter;
   else
      undeclutter;
   fi
done